import numpy as np
import csv
import random
from datetime import datetime
from time import strftime
from scipy.stats import scoreatpercentile
from dbfpy import dbf

'''
revision of test24
to calculate the false nagetive, false positive

'''

def getCurTime():
    """
    get current time
    Return value of the date string format(%Y-%m-%d %H:%M:%S)
    """
    format='%Y-%m-%d %H:%M:%S'
    sdate = None
    cdate = datetime.now()
    try:
        sdate = cdate.strftime(format)
    except:
        raise ValueError
    return sdate

def build_data_list(inputCSV):
    sKey = []
    fn = inputCSV
    ra = csv.DictReader(file(fn), dialect="excel")
    
    for record in ra:
        #print record[ra.fieldnames[0]], type(record[ra.fieldnames[-1]])
        for item in ra.fieldnames:
            temp = float(record[item])
            sKey.append(temp)
    sKey = np.array(sKey)
    sKey.shape=(-1,len(ra.fieldnames))
    return sKey

def build_satscanresult_dbf(inputDBF):
    sKey = np.array([])
    fn = inputDBF
    db = dbf.Dbf(fn)
    for record in db:
        temp = float(record[db.fieldNames[2]])
        if temp < (p + 0.00001):
            temp_id = int(float(record[db.fieldNames[0]]))
            sKey = np.append(sKey, temp_id)
            temp_id = int(float(record[db.fieldNames[1]]))
            sKey = np.append(sKey, temp_id)
            sKey = np.append(sKey, temp)
    sKey.shape = (-1, 3)
    return sKey

def build_satscanresult_pvalue(arrayLen, inputDBF):
    # build a list of pvalue, if not found in DBF, set as 1
    tPvalue = np.ones(arrayLen)
    tempResult = build_satscanresult_dbf(inputDBF)
    #i = 0
    for item in tempResult:
        tPvalue[int(item[0])] = item[2]
    return tPvalue

def build_unit_pvalue(inputCSV):
    unit_pvalue = []
    fn = inputCSV
    ra = csv.DictReader(file(fn), dialect="excel")
    
    for record in ra:
        #print float(record[ra.fieldnames[-1]]), type(record[ra.fieldnames[-1]])
        unit_pvalue.append(float(record[ra.fieldnames[pvalueLevel]]))
    return unit_pvalue

def cal_quantile(inputdata, q):
    temp = []
    #print len(inputdata[0])
    for i in range(0, len(inputdata[0])):
        #print scoreatpercentile(inputdata[:,0], 25)
        temp.append(scoreatpercentile(inputdata[:,i], int(q), limit = ()))
    return temp

def cal_quantile(inputdata, q):
    return  scoreatpercentile(inputdata, int(q), limit = ())

def cal_TP(unitPvalue, unitAttri, tempAreaTotal):
    tempMeasure = [0,0,0,0.0]
    i = 0
    for item in unitPvalue:
        if item < (p + 0.00001):
            if i in hot_16:
                tempMeasure[0] += unitAttri[i]
            else:
                tempMeasure[1] += unitAttri[i]
        i += 1
    tempMeasure[2] = tempAreaTotal - tempMeasure[0]
    tempMeasure[3] = tempMeasure[0]/(tempMeasure[1]+tempMeasure[2])
    return tempMeasure

#--------------------------------------------------------------------------
#MAIN

if __name__ == "__main__":
    print "begin at " + getCurTime()

    # old risk area
    '''
    mixed = [91,98,101,104,114,115,119,126,131,142,146,147,154,162,168,172]
    rural = [8,9,10,11,12,13,14,15,17,19,20,26,28,33,34,37]
    urban = [105,107,112,120,122,125,127,128,130,133,134,141,143,149,152,155]

    hot_1 = [9,130,147]
    hot_2 = hot_1 + [10,133,154]
    hot_4 = hot_2 + [12,17,125,131,141,146]
    hot_8 = hot_4 + [14,19,20,26,114,115,119,120,128,134,149,168]
    hot_16 = mixed + rural + urban
    '''

    hot_16 = [9, 10, 12, 13, 14, 17, 20, 23, 26, 27, 28, 33, 34, 35, 38, 41,
              95, 99, 109, 110, 114, 115, 117, 119, 125, 126, 127, 128, 130,
              131, 133, 134, 136, 137, 138, 139, 140, 141, 142, 143, 146, 147,
              149, 151, 152, 156, 157, 237 ]
    #unitCSV = "C:/TP1000_1m.csv"
    #unitCSV = 'C:/_DATA/CancerData/SatScan/mult6000/three16_format.csv'
    unitCSV = 'C:/_DATA/CancerData/SatScan/mult6000/three16_modify/three16_format_modify.csv'
    dataMatrix = build_data_list(unitCSV)  # [id, pop, cancer1, cancer2, cancer3]
    unit_attri = np.zeros((len(dataMatrix),4))
    
    # unit_attri [cancer, pop, rate, id]
    unit_attri[:,1] = dataMatrix[:,1]

    i = 0
    totalPop = 0
    for item in unit_attri[:,1]:
        if i in hot_16:
            totalPop += item
        i += 1

    #filePath = 'C:/_DATA/CancerData/SatScan/mult6000/three16_modify/satscan/'
    filePath = 'C:/_DATA/CancerData/SatScan/mult6000/three16_modify/ALK/SSD/'
    #filePath = 'C:/_DATA/CancerData/SatScan/mult6000/three16_modify/LLR_5p_improved/SSD/'
    #filePath = 'C:/_DATA/CancerData/SatScan/mult6000/three16_modify/LLR/SSD/'
    #filePath = 'C:/_DATA/CancerData/SatScan/mult6000/three16_modify/WARD/SSD/'
    #filePath = 'C:/_DATA/CancerData/SatScan/mult6000/three16_modify/CLK/SSD/'

    countMeasure = []

    pvalueList = range(1,20)
    pvalueLevel = -2 # -1: min pvalue, -2: last level
    for pvalueLevel in range(0, 2):
        pvalueLevel = - pvalueLevel -1
        print pvalueLevel
        countMeasure = []
        for p in pvalueList:
            p = 0.05 * p
            print p
            measure = []
            repeatTime = 1000
            case_measure = np.array([])
            pop_measure = np.array([])
            row = 0
            count = 0
            for repeat in range(0, repeatTime):
                #regionCSV = filePath + 'Full-Order-ALK_EBS_high_' + str(repeat) + "_pvalue.csv"
                #regionCSV = filePath + 'Full-Order-CLK_EBS_high_' + str(repeat) + "_pvalue.csv"
                #regionCSV = filePath + 'Full-Order-CLK_EBS_' + str(repeat) + "_whole_high_pvalue.csv"
                regionCSV = filePath + 'Full-Order-ALK_EBS_' + str(repeat) + "_whole_high_pvalue.csv"
                #regionCSV = filePath + 'WARD_EBS_' + str(repeat) + "_whole_high_pvalue.csv"
                #regionCSV = filePath + 'WARD_EBS_high_' + str(repeat) + "_pvalue.csv"
                #regionCSV = filePath + 'LLR_EBS_high_' + str(repeat) + "_pvalue.csv"
                #regionCSV = filePath + str(repeat) + ".gis.dbf"

                unit_attri[:,0] = dataMatrix[:,repeat + 2] # [cancer, pop, pvalue]
                unit_attri[:,2] = build_unit_pvalue(regionCSV)
                #unit_attri[:,2] = build_satscanresult_pvalue(len(unit_attri), regionCSV)

                i = 0
                totalCancer = 0
                for item in unit_attri[:,0]:
                    if i in hot_16:
                        totalCancer += item
                    i += 1
                tempMeasure = cal_TP(unit_attri[:,2], unit_attri[:,1], totalPop)    # pop
                measure.append(tempMeasure)
                tempMeasure = cal_TP(unit_attri[:,2], unit_attri[:,0], totalCancer)    # Cancer
                measure.append(tempMeasure)
                tempMeasure = cal_TP(unit_attri[:,2], np.ones(len(unit_attri)), len(hot_16))    # Cancer
                measure.append(tempMeasure)
            measure = np.array(measure)
            measure.shape = (repeatTime, -1)
            tempCountMeasure = [p, np.average(measure[:,-1]), cal_quantile(measure[:,-1], 25), np.median(measure[:,-1], axis=0), cal_quantile(measure[:,-1], 75)]
            fileLoc = filePath + 'significance_'+ str(-1-pvalueLevel) +'.csv'
            #np.savetxt(fileLoc, measure, delimiter=',', fmt = '%10.5f')
            countMeasure.append(tempCountMeasure)
        fileLoc = filePath + 'count_significance_'+ str(-1-pvalueLevel) +'.csv'
        np.savetxt(fileLoc, countMeasure, delimiter=',', fmt = '%10.5f')


    print "end at " + getCurTime()
    print "========================================================================"  

           
